#!/bin/env bash
#############################################################################
## Function: wget download hdfs files to localhost or browse hdfs directory.
## Warning: for big files, maybe you will block the network.
## Environment: run in the linux, and second sure you can visit the DSAP.
## Author: lijun06@58.com
## Ver: 0.2
## more info see: http://my.oschina.net/leejun2005
#############################################################################
	
. /etc/profile
startTime=`date +'%F %T'`
dsapDomain=dsap2.web.58dns.org

redEcho(){
	echo -e "\e[41;37;1m $1 \e[0m"  
}

greenEcho(){
	echo -e "\e[42;37;1m $1 \e[0m"
}

usage(){
	redEcho Usage:
	echo "bash wgetHdfs2.sh [-l|-d] hdfsFullPath"
	echo "-l: list hdfs directory files"
	echo "-d: download hdfs directory files or single file"
	echo "hdfsFullPath: directory or file's HDFS full path"
	echo "Example: bash wgetHdfs2.sh -l /tmp/dsap/rawdata/lijun06/RandShow58"
	echo "Example: bash wgetHdfs2.sh -d /tmp/dsap/rawdata/lijun06/RandShow58"
	echo "Example: bash wgetHdfs2.sh -d /tmp/dsap/rawdata/lijun06/RandShow58/xdm"
	redEcho Warning:
	echo "for big files, maybe you will block the network."
	exit 126
}

#检查 DSAP 网络是否正常
checkNetwork(){
	ping -c 2 -i 0.5 -W 2 $dsapDomain &>/dev/null || { redEcho "Network Exception: $dsapDomain" && exit 1; }
}
	
#根据 hdfs 路径生成 HTTP url
genReqUrl(){
	#reqUrl='http://dsap2.web.58dns.org/dsap-agent/?targetserver=db-cdh-197&targetport=50075&targetpage=browseDirectory&targetprefix=jsp&dir='$baseHdfsPath'&namenodeInfoPort=50070&nnaddr=10.5.12.196:9000'
	reqUrl='http://dsap2.web.58dns.org/dsap-agent/?targetserver=db-cdh-216&targetport=50075&targetpage=browseDirectory&targetprefix=jsp&dir='$baseHdfsPath'&namenodeInfoPort=50070&nnaddr=10.5.12.196:9000'
}

#列出 hdfs 目录
listDir(){
	genReqUrl $baseHdfsPath
	nameList=`wget "$reqUrl" -O- 2>/dev/null|grep -Po '(?<=:9000">)[^/]*?(?=</a></B><br></td>)'|sed '/^_/d'`
	while read name
	do
		echo $name
	done< <(echo "$nameList")
}

#下载 hdfs 文件，支持单个文件或者目录，不考虑递归
downLoadFiles(){
	listDir
	# 下载单个文件
	if [[ "$nameList" == "" ]]
	then
		tmpUrl="http://10.5.12.216:50075/streamFile$baseHdfsPath?nnaddr=10.5.12.196:9000"
		serverFileName=`curl -I "$tmpUrl" 2>/dev/null|awk -F\" '/attachment/{print $(NF-1)}'`
		saveName=`echo "$baseHdfsPath"|awk -F/ '{print $(NF-2)"_"$(NF-1)"_"$NF}'`
		[[ "$serverFileName" != "$baseHdfsPath" ]] && redEcho "Download Path is input Error: $tmpUrl" && exit 125
		wget "$tmpUrl" -O $saveName && greenEcho "$saveName >> : download success!" || redEcho "$saveName << : download Fail!"
	else	#下载目录，需要是最底层目录，不考虑递归
		while read name
		do
			downFileFullName=$baseHdfsPath/$name
			tmpUrl="http://10.5.12.216:50075/streamFile$downFileFullName?nnaddr=10.5.12.196:9000"
			serverFileName=`curl -I "$tmpUrl" 2>/dev/null|awk -F\" '/attachment/{print $(NF-1)}'`
			saveName=`echo "$downFileFullName"|awk -F/ '{print $(NF-2)"_"$(NF-1)"_"$NF}'`
			[[ "$serverFileName" != "$downFileFullName" ]] && redEcho "Download Path is input Error: $tmpUrl" && exit 125
			wget "$tmpUrl" -O $saveName && greenEcho "$name >> : download success!" || redEcho "$saveName << : download Fail!"
		done< <(echo "$nameList")
	fi
} 

optionParam=$1
baseHdfsPath=$2
echo $optionParam|grep -qE '^(-d|-l)$' || usage
echo $baseHdfsPath|grep -qE '^/' || usage

# DSAP 网络检查，自行决定是否开启检查，默认开启
checkNetwork

if [[ $optionParam == "-l" ]]
then
	echo --------------------$startTime----------------------
	listDir "$baseHdfsPath"
	echo --------------------`date +'%F %T'`----------------------
else
	echo --------------------$startTime----------------------
	downLoadFiles "$baseHdfsPath"
	echo --------------------`date +'%F %T'`----------------------
fi


